# -*- coding: utf-8 -*-
# @Time    : 2020/1/2 18:02
# @Author  : lzg
# @FileName: backtest.py
# @Software: PyCharm
import pandas as pd
import numpy as np
import h2o
import datetime
import time
import matplotlib.pyplot as plt
# opencost,closecost,closecost2 are all numeric scalar
# output is a value calculate by simple interest
# DTtradep应该是代表一笔交易的数据
class backtest_minutes:

    def __init__(self, TimestampSignal, TimestampPrice, opencost, closecost, closecost2):
        '''
        :param TimestampSignal:
        :param TimestampPrice:
        :param opencost: 开仓成本
        :param closecost: 平仓成本
        :param closecost2: 日内平仓成本
        '''
        self.TimestampSignal = TimestampSignal
        self.TimestampPrice = TimestampPrice
        self.opencost = opencost
        self.closecost = closecost
        self.closecost2 = closecost2

    # calculate the returns from long operations
    # given open price, close price, cost of open the position, cost for close the position
    def long(self, openprice, closeprice, opencost, closecost):
        '''
        多头收益，close/open基础上加上成本
        '''
        return ((closeprice * (1 - closecost)) / (openprice * (1 + opencost)) - 1)

    # calculate the returns from short operations
    # given open price, close price, cost of open the position, cost for close the position
    #做空最开始拿到一笔钱 减去 平仓的成本加费用
    def short(self, openprice, closeprice, opencost, closecost):
        nominator = openprice * (1 - opencost) - closeprice * (1 + closecost)
        denominator = openprice * (1 - opencost)
        return (nominator / denominator)

    # r is the stoploss criteria, r is the loss from opening point
    # DTtradep is a pandas dataframe, stands for datetime and trading price
    # the first column of DTtradep is a 'DATETIME', which is a timestamp object
    # the second column s a 'close', which is a float64 object
    # opencost is the cost for open
    # closecost is the normal close cost
    # closecost2 is the intraday cost

    # crossday_long_with_stoploss and crossday_short_with_stoploss
    # tries to calculate the returns for any time (intraday or not)
    # e.g. the returns between '2010-09-10 13:47:00' and '2010-09-20 09:56:00'
    # if stoploss happens, then we use the next minutes bar as the price for closing the position
    # if the stoploss did not happen, then we use the last element of tradep as
    # the price for closing the position

    # the function returns two values in list form
    # (1) a dataframe with first column being datetime.date, it can be duplicated.
    # as many transactions can be made on a single day.
    # e.g. many times of stoploss, close the position at a day open. reopen in the mid of the day
    # second column being returns of a set of these transactions
    # (2) the second value is the ending datetime
    #针对有可能跨天的交易，一般都会当天结算
    #r代表止损率
    #interdayes代表一笔交易跨度的天数
    def crossday_long_with_stoploss(self, r, DTtradep, opencost, closecost, closecost2, openprice='close',
                                    closeprice='close'):
        minr = np.where(DTtradep['low'] < DTtradep[openprice].iloc[0] * (1 - r))[0]
        tradingvalue = []
        if len(minr) != 0:  # stoploss occured
            # get the rows before the stoploss time
            interprice = DTtradep[:(minr[0] + 1)]
            interdates = interprice['Date'].unique()
            print("long stoploss occured! " + str(interprice['DATETIME'].iloc[-1]) + "\n")
            STOPLOSS = True
        else:  # if no stoploss happens
            interprice = DTtradep
            interdates = interprice['Date'].unique()
            STOPLOSS = False

        # if this is a intraday trade
        if (len(interdates) < 2):
            if STOPLOSS == True:
                #有止损，就亏了百分之r
                tradingvalue.append(-r)
            else:
                tradingvalue.append(
                    self.long(interprice[openprice].iloc[0], interprice[closeprice].iloc[-1], opencost, closecost2))

        # if this is not a intraday trade
        if (len(interdates) >= 2):
            # get the first day of trading return
            # because the cost of opening, we have to do that seperate
            #当日结算，开仓价格与开仓日收盘价格做一个结算，因为没有实际平仓，无平仓成本
            closeprev = interprice[openprice].iloc[0]
            closethatday = interprice['close'][interprice['Date'] == interdates[0]].as_matrix()[-1]
            tradingvalue.append(self.long(closeprev, closethatday, opencost, 0))

            # if tradingvalue[0]<0:
            #    opennextday=interprice['close'][interprice['Date']==interdates[1]].as_matrix()[0]
            #    tradingvalue.append(self.long(closethatday,opennextday,0,closecost))
            #    datevalues=pd.DataFrame([interdates[:2],tradingvalue]).transpose()
            #    datevalues.columns=['Date','Returns']
            #    endperiod=interprice['DATETIME'][interprice['Date']==interdates[1]].iloc[0]-datetime.timedelta(minutes=1)
            #    return [datevalues,endperiod]

            for i in range(1, len(interdates) - 1):
                #  get the close price previous day
                closeprev = interprice['close'][interprice['Date'] == interdates[(i - 1)]].as_matrix()[-1]
                # get the close price today
                closethatday = interprice['close'][interprice['Date'] == interdates[i]].as_matrix()[-1]
                tradingvalue.append(self.long(closeprev, closethatday, 0, 0))

            #到了平仓这一天做一个判断，是否是止损
            #因为是按日结算的，所以要取昨天的收盘价与今天的平仓价做日收益结算
            if STOPLOSS == True:
                closeprev = interprice['close'][interprice['Date'] == interdates[len(interdates) - 2]].as_matrix()[0]
                closethatday = interprice[openprice].as_matrix()[0] * (1 - r)
                tradingvalue.append(self.long(closeprev, closethatday, 0, closecost))
            else:
                # get the last day of trading return
                # because the cost of closing, we have to do that seperate
                closeprev = interprice['close'][interprice['Date'] == interdates[len(interdates) - 2]].as_matrix()[-1]
                closethatday = interprice[closeprice].as_matrix()[-1]
                tradingvalue.append(self.long(closeprev, closethatday, 0, closecost))

        datevalues = pd.DataFrame([interdates, tradingvalue]).transpose()
        datevalues.columns = ['Date', 'Returns']

        endperiod = interprice['DATETIME'].iloc[-1]

        return [datevalues, endperiod]

    def crossday_short_with_stoploss(self, r, DTtradep, opencost, closecost, closecost2, openprice='close',
                                     closeprice='close'):
        maxr = np.where(DTtradep['high'] > DTtradep['close'].iloc[0] * (1 + r))[0]
        tradingvalue = []
        if len(maxr) != 0:  # stoploss occured
            # get the rows before the stoploss time
            interprice = DTtradep[:(maxr[0] + 1)]
            interdates = interprice['Date'].unique()
            print("short stoploss occured! " + str(interprice['DATETIME'].iloc[-1]) + "\n")
            STOPLOSS = True
        else:  # if no stoploss happens
            interprice = DTtradep
            interdates = interprice['Date'].unique()
            STOPLOSS = False

        # if this is a intraday trade
        if (len(interdates) < 2):
            if STOPLOSS == True:
                tradingvalue.append(-r)
            else:
                tradingvalue.append(
                    self.short(interprice[openprice].iloc[0], interprice[closeprice].iloc[-1], opencost, closecost2))

        # if this is not a intraday trade
        if (len(interdates) >= 2):
            # get the first day of trading return
            # because the cost of opening, we have to do that seperate
            closeprev = interprice[openprice].iloc[0]
            closethatday = interprice['close'][interprice['Date'] == interdates[0]].as_matrix()[-1]
            tradingvalue.append(self.short(closeprev, closethatday, opencost, 0))

            # if tradingvalue[0]<0:
            #    opennextday=interprice['close'][interprice['Date']==interdates[1]].as_matrix()[0]
            #    tradingvalue.append(self.short(closethatday,opennextday,0,closecost))
            #    datevalues=pd.DataFrame([interdates[:2],tradingvalue]).transpose()
            #    datevalues.columns=['Date','Returns']
            #    endperiod=interprice['DATETIME'][interprice['Date']==interdates[1]].iloc[0]-datetime.timedelta(minutes=1)
            #    return [datevalues,endperiod]

            for i in range(1, len(interdates) - 1):
                #  get the close price previous day
                closeprev = interprice['close'][interprice['Date'] == interdates[(i - 1)]].as_matrix()[-1]
                # get the close price today
                closethatday = interprice['close'][interprice['Date'] == interdates[i]].as_matrix()[-1]
                tradingvalue.append(self.short(closeprev, closethatday, 0, 0))

            if STOPLOSS == True:
                closeprev = interprice['close'][interprice['Date'] == interdates[len(interdates) - 2]].as_matrix()[0]
                closethatday = interprice[openprice].as_matrix()[0] * (1 + r)
                tradingvalue.append(self.short(closeprev, closethatday, 0, closecost))
            else:
                # get the last day of trading return
                # because the cost of closing, we have to do that seperate
                closeprev = interprice['close'][interprice['Date'] == interdates[len(interdates) - 2]].as_matrix()[-1]
                closethatday = interprice[closeprice].as_matrix()[-1]
                tradingvalue.append(self.short(closeprev, closethatday, 0, closecost))

        datevalues = pd.DataFrame([interdates, tradingvalue]).transpose()
        datevalues.columns = ['Date', 'Returns']

        endperiod = interprice['DATETIME'].iloc[-1]

        return [datevalues, endperiod]

        # find the first set of transaction: buy and then sell, or short and then cover

    # i.e. find the first example of pair 1,-1 or -1,1
    #有一连串的交易信号，本质上是找到第一组非零的交易信号，即第一组1，-1
    # if either actionindex1=='end' or actionindex2=='end', then we shouldn't preceed forward
    # action is a munpy array or a pandas series
    def trading_index_finder(self, actions):
        # get the index for the first nonzero
        try:
            actionindex1 = np.where(actions != 0)[0][0]
        except:
            #没有发生交易，所有都标记为end
            actionindex1 = 'end'
            actionindex2 = 'end'

        try:
            if (actions.iloc[actionindex1] == 1):
                actionindex2 = np.where(actions == -1)[0]
                actionindex2 = actionindex2[actionindex2 > actionindex1][0]

            if (actions.iloc[actionindex1] == -1):
                actionindex2 = np.where(actions == 1)[0]
                actionindex2 = actionindex2[actionindex2 > actionindex1][0]
        except:
            actionindex2 = 'end'

        #如果最后的actionindex2是end，那么说明这笔交易是未完成的
        return [actionindex1, actionindex2]

        # input, KnownState is a dataframe with four columns

    # DATETIME(pd.timestamp), Date(datetime.date), Time(datetime.time), Prediction(float64)
    # find the 25 75 quantile for a set of prediction
    # 算历史预测值的百分位，大于75%百分位做多，小于25%百分位做空，在中间就维持原来的操作
    def get_quantile_by_time(self, KnownState, quantile1=25, quantile2=75):
        Qall = []
        TimestampLevels = KnownState['Time'].unique()
        for Tstamp in TimestampLevels:
            Qall.append(np.percentile(KnownState['Prediction'][KnownState['Time'] == Tstamp],
                                      [quantile1, quantile2]).reshape(1, 2))
        return np.concatenate(Qall)

    # input, TimestampSignal is a dataframe with four columns
    # DATETIME(pd.timestamp), Date(datetime.date), Time(datetime.time), Prediction(float64)
    # startn is the days that we get started，为计算百分位预留一段时间的样本，获得样本各个时间点的百分位划分
    # endn则代表实际要用的数据时间长度
    # generate an actionlist of 1(buy,cover), 0(maintain the position before), -1(short,sell)
    # for example, if we are standing at 2007-07-05, then we using all the data backing to the first day of our prediction
    # and then we use all the Predictions between the first day and 2007-07-04 to calculate a 25 75 quantile
    # and then we examine if the 2007-07-05 is greater than the upper 25 quantile, if yes, write 1
    # similiarly ,if the 2007-07-05 is smaller than the lower 25 quantile, if yes, write -1
    # the output is DATETIME(pd.timestamp), Date(datetime.date), Time(datetime.time), Prediction(1,0,-1)
    def actionlist_generator(self, startn=13, lookbackn=None, Timeeffect=True):
        self.actionlist = []
        #算出每天大概有几笔交易，然后作为步长（代表进行了一天的交易）
        stepsize = round(self.TimestampSignal.shape[0] / len(self.TimestampSignal['Date'].unique()))

        # remove the Date that has different stepsize
        if Timeeffect == True:
            for Date in self.TimestampSignal['Date'].unique():
                if sum(self.TimestampSignal['Date'] == Date) != stepsize:
                    self.TimestampSignal = self.TimestampSignal[self.TimestampSignal['Date'] != Date]
                    print(str(Date) + " has been removed because of missing data(prediction)!\n")

        startpoint = startn * stepsize
        endpoint = int(self.TimestampSignal.shape[0] / stepsize) * stepsize - stepsize
        for i in range(startpoint, (endpoint + 1), stepsize):
            if lookbackn == None:
                KnownState = self.TimestampSignal[:i]
            else:
                lookbackn = lookbackn * stepsize
                KnownState = self.TimestampSignal[(i - lookbackn):i]

            Qall = self.get_quantile_by_time(KnownState, quantile1=25, quantile2=75)

            if Timeeffect == False:
                Qall = np.mean(Qall, axis=0)

            # find that day (both prediction and mins data)
            PredictionThatday = self.TimestampSignal[i:(i + stepsize)]
            print(PredictionThatday['Date'].iloc[0])
            PredictionThatday = PredictionThatday['Prediction'].copy().as_matrix()

            if Timeeffect == False:
                # if we will need to do anything that day
                PredictionThatday[PredictionThatday < Qall[0]] = -1
                PredictionThatday[PredictionThatday > Qall[1]] = 1
                PredictionThatday[(PredictionThatday > Qall[0]) &
                                  (PredictionThatday < Qall[1])] = 0
            else:
                # if we will need to do anything that day
                PredictionThatday[PredictionThatday < Qall[:, 0]] = -1
                PredictionThatday[PredictionThatday > Qall[:, 1]] = 1
                PredictionThatday[(PredictionThatday > Qall[:, 0]) &
                                  (PredictionThatday < Qall[:, 1])] = 0

            self.actionlist.append(PredictionThatday)

        self.actionlist = np.concatenate(self.actionlist)
        DATETIME = self.TimestampSignal[['DATETIME', 'Date', 'Time']][startpoint:(endpoint + stepsize)]
        DATETIME['Prediction'] = self.actionlist
        self.actionlist = DATETIME

    # to make decision every 30 mins and the dicided if we need to hold the position for another day
    # r is how much percentage rate do we need to do stoploss
    # Type is LongOnly ShortOnly LongShort
    def crossday_fixedstoploss_mins_value_everyday(self, r, Type='LongShort'):
        self.datevalues = pd.DataFrame()

        # generate a actionlist of 1(buy,cover), 0(maintain the position before), -1(short,sell)
        try:
            actionlist = self.actionlist
        except:
            self.actionlist_generator(startn=13, Timeeffect=False)
            actionlist = self.actionlist

        while (len(actionlist) > 0):

            # find the first set of transaction: buy and then sell, or short and then cover
            if Type == 'LongShort':
                [actionindex1, actionindex2] = self.trading_index_finder(actionlist['Prediction'])

            if Type == 'LongOnly':
                [actionindex1, actionindex2] = self.trading_index_finder_longonly(actionlist['Prediction'])

            # that is the end of a loop, break out of the loop
            if ((actionindex1 == 'end') or (actionindex2 == 'end')):
                break

            [Datetime1, Action1] = actionlist[['DATETIME', 'Prediction']].iloc[actionindex1]
            [Datetime2, Action2] = actionlist[['DATETIME', 'Prediction']].iloc[actionindex2]

            # get the trading price
            DTtradep = self.TimestampPrice[
                (self.TimestampPrice['DATETIME'] >= Datetime1) & (self.TimestampPrice['DATETIME'] <= Datetime2)]
            DTtradep['close'] = DTtradep['close'].astype(float)

            if (Action1 == 1):
                if Type == 'LongOnly' or Type == 'LongShort':
                    payoff = self.crossday_long_with_stoploss(r, DTtradep, self.opencost, self.closecost,
                                                              self.closecost2)
                    self.datevalues = self.datevalues.append(payoff[0])
                else:
                    payoffvalue = pd.DataFrame(
                        [DTtradep['Date'].unique(), [0] * len(DTtradep['Date'].unique())]).transpose()
                    payoffvalue.colnames = ['Date', 'Returns']
                    payoff = [payoffvalue, DTtradep['DATETIME'].iloc[-1]]

            if (Action1 == (-1)):
                if Type == 'ShortOnly' or Type == 'LongShort':
                    payoff = self.crossday_short_with_stoploss(r, DTtradep, self.opencost, self.closecost,
                                                               self.closecost2)
                    self.datevalues = self.datevalues.append(payoff[0])
                else:
                    payoffvalue = pd.DataFrame(
                        [DTtradep['Date'].unique(), [0] * len(DTtradep['Date'].unique())]).transpose()
                    payoffvalue.colnames = ['Date', 'Returns']
                    payoff = [payoffvalue, DTtradep['DATETIME'].iloc[-1]]

                    # update the actionlist

            #把上笔统计好的交易的数据全部删掉，不停的循环往复，直到全部算完
            actionlist = actionlist[actionlist['DATETIME'] > payoff[1]]
            print(payoff[1])

        return self.datevalues


### class of the features for using prices (only using prices)
### the inputs for short_term_prices would be a dataframe with Date (must),Time(must),open,high,low,close(must)
###  (all of them is minutes/seconds data)
### the timestamp referred is a string
#用于获取半小时时间段的收盘价数据
class within_day_price_features:

    #   Datetime_Op=Datetime_Operation()
    #   self.data['Time']=Datetime_Op.date_to_timestamp(self.data['Time'])
    #   self.data['Date']=Datetime_Op.date_to_timestamp(self.data['Date'])
    def __init__(self, data):
        self.data = data

    # date and time is datetime64[ns] object
    # timestamp1 is a string, so is timestamp2
    # timestamp2 is greater than or equal to timestamp1
    def returns_without_cost(self, timestamp1, timestamp2, direction):
        close_prices = self.get_close_price(timestamp1, timestamp2)
        price1 = close_prices[0]
        price2 = close_prices[-1]
        if direction > 0:
            self.long_return = price2 / price1 - 1
            return self.long_return
        else:
            self.short_return = (price1 - price2) / price1
            return self.short_return

    # get the close, open, high, low, price of certain time interval  (inclusive)
    # timestamp1 is a string, so is timestamp2
    # timestamp2 is greater than or equal to timestamp1
    def get_close_price(self, timestamp1, timestamp2):
        close_prices = self.data['close'][(self.data['Time'] >= pd.Timestamp(timestamp1).time()) & (
                    self.data['Time'] <= pd.Timestamp(timestamp2).time())]
        return close_prices.as_matrix()

    # max price/open price -1
    # timestamp1 is a string, so is timestamp2
    # timestamp2 is greater than or equal to timestamp1
    def maxprice_to_open(self, timestamp1, timestamp2):
        prices = self.get_close_price(timestamp1, timestamp2)
        maxprice_open = prices.max() / prices[0] - 1
        return maxprice_open

    # min price/open price -1
    # timestamp1 is a string, so is timestamp2
    # timestamp2 is greater than or equal to timestamp1
    def minprice_to_open(self, timestamp1, timestamp2):
        prices = self.get_close_price(timestamp1, timestamp2)
        minprice_open = prices.min() / prices[0] - 1
        return minprice_open

    # get the close price to settle price
    def close_to_settle(self):
        prices = self.get_close_price('14:00:00', '15:00:00')
        closesettle = prices[-1] / prices.mean() - 1
        return closesettle


class H2Odeeplearning:

    def __init__(self, data):
        self.data = data
        self.data = self.data.reset_index(drop=True)
        self.data = self.data.rename(columns={self.data.columns.values[0]: 'Returns'}, inplace=False)
        self.data = self.data.rename(columns={self.data.columns.values[1]: 'Date'}, inplace=False)

        self.data.to_csv(r'D:\work\ZNC\AI\CTA\test_data.csv')

        # transform a data frame to a h2o frame
        self.h2odata = h2o.H2OFrame.from_python(self.data, column_names=list(self.data.columns))
        self.h2odata = self.h2odata[1:self.h2odata.shape[0], :]

    # this function makes prediction once everyday based on a data set
    # as it's name calls, it's online learning.
    # once a model is trained, it then make predictions on the following day
    # it rolls forward until the very end
    # There must be one row called 'Date', say '2007-06-17', a datetime.date() object
    # There must be another row called 'Returns', say 0.015,
    # that is, the return for certain time interval for a single day,
    # say in 2007-06-17, the return from 11:15 to 15:00 is 0.015
    # the framework of the autoencoder is based on
    # https://github.com/h2oai/h2o-3/blob/master/h2o-py/tests/testdir_algos/deeplearning/pyunit_autoencoderDeepLearning_large.
    #线上学习的一个模式
    #insample_date_until开始的时间
    def online_dl_with_autoencoder_once_a_day(self, insample_date_until, HiddenLayer=[7, 5, 3],
                                              HiddenDropout=[0.2, 0.2, 0.2]):

        # get the index for that day
        try:
            start_index = pd.Index(self.data["Date"]).get_loc(pd.to_datetime(insample_date_until).date())
        except:
            print("This date is not a valid trading date, please enter another date \n")

        # result container
        result = pd.DataFrame()
        # define the pretrained model
        # (autoencoder for weight initialization)
        #预训练一个自编码器
        #金融数据一般比较小，不易收敛，所以用自编码器加快收敛的速度
        pretrainedmodel = h2o.estimators.H2OAutoEncoderEstimator(
            activation="Tanh",
            model_id="pretrainedmodel",
            hidden=HiddenLayer,
            hidden_dropout_ratios=HiddenDropout,
            input_dropout_ratio=0,
            standardize=True,
            seed=225,
            train_samples_per_iteration=-1,#拿所有样本进行训练
            epochs=30)

        # define the deep learning model
        #训练一个全连接神经网络
        model = h2o.estimators.H2ODeepLearningEstimator(
            activation="Tanh",
            pretrained_autoencoder="pretrainedmodel",
            model_id="model",
            hidden=HiddenLayer,
            hidden_dropout_ratios=HiddenDropout,
            input_dropout_ratio=0,
            standardize=True,
            seed=225,
            train_samples_per_iteration=-1,
            epochs=30)

        for i in range(start_index, self.data.shape[0]):
            # add a decay factor to the h2o frame
            # h2odatadecay=h2o.H2OFrame.from_python(np.exp(np.arange(i)[::-1]*-0.005).reshape(i,1))

            # train the pretrained model
            #用0-i的数据预测i+1
            pretrainedmodel.train(
                x=list(self.data.columns.difference(["Returns", "Date"])),
                training_frame=self.h2odata[range(0, i), :])  # *h2odatadecay)
            # validation_frame=self.h2odata[i,:])

            # train the deep learning model
            model.train(
                x=list(self.data.columns.difference(["Returns", "Date"])),
                y="Returns",
                training_frame=self.h2odata[range(0, i), :])  # *h2odatadecay)
            # validation_frame=self.h2odata[i,:])

            # make prediction based on all the remaining data
            pred = model.predict(self.h2odata[i, :])
            pred = pred.as_data_frame()
            pred = pd.DataFrame([self.data.iloc[i]["Date"], pred.values[0][0]]).transpose()
            pred.columns = ["Date", "Prediction"]
            result = result.append(pred, ignore_index=True)
            print(pred)

        return result


## combine value and date
# mainly to combine value after backtest (either for compound or single)
# the value always start from 1, i.e. the position is the last time point before trading
# however, the date and time is from the first day of trading
# when it's a operation, we use _ to seperate each word, say getdata_RosefinchSQL
# when it's a value, we use Capital Letter to seperate each word
# date is a numpy array with datetime object in it
# value is a numeric value
#因为回测总是从1开始，所以第一个值对我们是没有用处的，把values的第一个值删掉
class combine_date_value:

    def __init__(self, date, value):
        # to omit the first value 1
        try:
            self.value = value[1:].as_matrix()
        except:
            self.value = value[1:]
            print('value does not need to be convert to numpy array, it is already is! \n')

        try:
            self.date = date[:len(self.value)].as_matrix()
        except:
            self.date = date[:len(self.value)]
            print('dates do not need to be convert to numpy array, it is already is! \n')

    def date_and_value(self, include_first=True, colnames=['date', 'value']):
        if include_first == True:
            test = self.date[0]
            delta = datetime.timedelta(1)
            PrevDay = self.date[0] - datetime.timedelta(1)
            self.date = np.insert(self.date, 0, PrevDay)
            self.value = np.insert(self.value, 0, 1)
            self.datevalue = pd.DataFrame([self.date, self.value]).transpose()
            self.datevalue.columns = colnames
        else:
            self.datevalue = pd.DataFrame([self.date, self.value]).transpose()
            self.datevalue.columns = colnames

#如果一天之内发生多笔交易，把多笔交易的payoff合在一起
#日内的payoff加在一起
class Returns_calculation:

    # returns is a data frame with first row being date, second row being Returns
    def __init__(self, returns):
        self.returns = returns
        self.returns.columns = ['Date', 'Returns']

    # merge_by_date is to sum over returns of the same date
    def merge_by_date(self):
        MergedReturns = self.returns.groupby(self.returns['Date'])
        MergedReturns = MergedReturns.sum()
        MergedReturns['Date'] = MergedReturns.index
        self.returns = MergedReturns

    # impute_zero_returns is to compare the trade record with the trading dates
    # to find the trading dates with zero returns and fill it in
    # tradedates are a pandas series
    #填充空仓天数
    def impute_zero_returns(self, tradedates):
        if (type(tradedates) is np.ndarray):
            tradedates = tradedates.reshape(len(tradedates), 1)
            tradedates = pd.DataFrame(tradedates)
            tradedates.columns = ['Date']
        try:
            self.returns = tradedates.merge(self.returns, on='Date', how='left')
            self.returns = self.returns.fillna(0)
        except:
            print('Insert 0 failed \n')

    # add up the returns together to calculate the simple returns
    #无论如何都用一块钱进行交易，所以我们的return可以加在一起，不存在复利的情况
    def returns_to_Simplevalues(self):
        self.SimpleValue = np.cumsum(np.insert(self.returns['Returns'].as_matrix(), 0, 1))

# split by date to get a object containing everyday mins data
# return a list containing all the single day dataframe
# 'Time', 'Date', 'DATETIME','preclose','open','high','low','close','amt','volumn'
#创造一个list，所有的element是每个交易日的数据
def split_by_day(data):
    data['Time'], data['Date'] = data['DATETIME'].apply(lambda x: x.time()), data['DATETIME'].apply(lambda x: x.date())
    oneday = []
    for dates in data['Date'].unique():
        oneday.append(data[data['Date'] == dates])
    return oneday


########################### make train using classes ##########################
def maketrain(oneday):
    Date = oneday['Date'].iloc[0]
    oneday_price = within_day_price_features(oneday[['Date', 'Time', 'open', 'high', 'low', 'close']])

    #半小时的return
    start1000 = oneday_price.returns_without_cost("09:31:00", "10:00:00", 1)
    start1030 = oneday_price.returns_without_cost("10:00:00", "10:30:00", 1)
    start1100 = oneday_price.returns_without_cost("10:30:00", "11:00:00", 1)
    start1300 = oneday_price.returns_without_cost("11:00:00", "13:00:00", 1)
    start1330 = oneday_price.returns_without_cost("13:00:00", "13:30:00", 1)
    start1400 = oneday_price.returns_without_cost("13:30:00", "14:00:00", 1)
    start1430 = oneday_price.returns_without_cost("14:01:00", "14:30:00", 1)
    start1500 = oneday_price.returns_without_cost("14:30:00", "15:00:00", 1)

    #预测目标，当前到收盘的return
    R1000 = float(oneday_price.returns_without_cost("10:00:00", "15:00:00", 1))
    R1030 = float(oneday_price.returns_without_cost("10:30:00", "15:00:00", 1))
    R1100 = float(oneday_price.returns_without_cost("11:00:00", "15:00:00", 1))
    R1300 = float(oneday_price.returns_without_cost("13:00:00", "15:00:00", 1))
    R1330 = float(oneday_price.returns_without_cost("13:30:00", "15:00:00", 1))
    R1400 = float(oneday_price.returns_without_cost("14:00:00", "15:00:00", 1))
    R1430 = float(oneday_price.returns_without_cost("14:30:00", "15:00:00", 1))

    #时间段最高价除以第一分钟收盘价
    maxpT1000 = oneday_price.maxprice_to_open("09:31:00", "10:00:00")
    maxpT1030 = oneday_price.maxprice_to_open("09:31:00", "10:30:00")
    maxpT1100 = oneday_price.maxprice_to_open("09:31:00", "11:00:00")
    maxpT1300 = oneday_price.maxprice_to_open("09:31:00", "13:00:00")
    maxpT1330 = oneday_price.maxprice_to_open("09:31:00", "13:30:00")
    maxpT1400 = oneday_price.maxprice_to_open("09:31:00", "14:00:00")
    maxpT1430 = oneday_price.maxprice_to_open("09:31:00", "14:30:00")

    minpT1000 = oneday_price.minprice_to_open("09:31:00", "10:00:00")
    minpT1030 = oneday_price.minprice_to_open("09:31:00", "10:30:00")
    minpT1100 = oneday_price.minprice_to_open("09:31:00", "11:00:00")
    minpT1300 = oneday_price.minprice_to_open("09:31:00", "13:00:00")
    minpT1330 = oneday_price.minprice_to_open("09:31:00", "13:30:00")
    minpT1400 = oneday_price.minprice_to_open("09:31:00", "14:00:00")
    minpT1430 = oneday_price.minprice_to_open("09:31:00", "14:30:00")

    #计算收盘价\结算价
    prevClose_prevSettle = oneday_price.close_to_settle()
    OpenPrice = oneday_price.get_close_price("09:31:00", "15:00:00")[0]
    ClosePrice = oneday_price.get_close_price("09:31:00", "15:00:00")[-1]

    #得到一个n*1的矩阵
    train = pd.DataFrame([Date, OpenPrice, ClosePrice, prevClose_prevSettle,
                          start1000, start1030, start1100, start1300, start1330, start1400, start1430, start1500,
                          R1000, R1030, R1100, R1300, R1330, R1400, R1430,
                          maxpT1000, maxpT1030, maxpT1100, maxpT1300, maxpT1330, maxpT1400, maxpT1430,
                          minpT1000, minpT1030, minpT1100, minpT1300, minpT1330, minpT1400, minpT1430]).transpose()
    return train

def combine_datetime(date, time):
    Datetime = []
    for i in range(len(date)):
        Datetime.append(datetime.datetime.combine(date.iloc[i], time.iloc[i]))
    return Datetime


def add_time(prediction, time):
    prediction['Time'] = [time] * prediction.shape[0]
    return prediction
def split_by_day(data):
    """
    split by date to get a object containing everyday mins data
    return a list containing all the single day dataframe
    'Time', 'Date', 'DATETIME','preclose','open','high','low','close','amt','volumn'
    """
    data['Time'], data['Date'] = data['DATETIME'].apply(lambda x: x.time()), \
                                 data['DATETIME'].apply(lambda x: x.date())
    oneday = []
    for dates in data['Date'].unique():
        oneday.append(data[data['Date'] == dates])
    return oneday


def maketrain(oneday):
    Date = oneday['Date'].iloc[0]
    oneday_price = within_day_price_features(
        oneday[['Date', 'Time', 'open', 'high', 'low', 'close']])

    start1000 = oneday_price.returns_without_cost("09:31:00", "10:00:00", 1)
    start1030 = oneday_price.returns_without_cost("10:00:00", "10:30:00", 1)
    start1100 = oneday_price.returns_without_cost("10:30:00", "11:00:00", 1)
    start1300 = oneday_price.returns_without_cost("11:00:00", "13:00:00", 1)
    start1330 = oneday_price.returns_without_cost("13:00:00", "13:30:00", 1)
    start1400 = oneday_price.returns_without_cost("13:30:00", "14:00:00", 1)
    start1430 = oneday_price.returns_without_cost("14:01:00", "14:30:00", 1)
    start1500 = oneday_price.returns_without_cost("14:30:00", "15:00:00", 1)

    R1000 = float(oneday_price.returns_without_cost("10:00:00", "15:00:00", 1))
    R1030 = float(oneday_price.returns_without_cost("10:30:00", "15:00:00", 1))
    R1100 = float(oneday_price.returns_without_cost("11:00:00", "15:00:00", 1))
    R1300 = float(oneday_price.returns_without_cost("13:00:00", "15:00:00", 1))
    R1330 = float(oneday_price.returns_without_cost("13:30:00", "15:00:00", 1))
    R1400 = float(oneday_price.returns_without_cost("14:00:00", "15:00:00", 1))
    R1430 = float(oneday_price.returns_without_cost("14:30:00", "15:00:00", 1))

    maxpT1000 = oneday_price.maxprice_to_open("09:31:00", "10:00:00")
    maxpT1030 = oneday_price.maxprice_to_open("09:31:00", "10:30:00")
    maxpT1100 = oneday_price.maxprice_to_open("09:31:00", "11:00:00")
    maxpT1300 = oneday_price.maxprice_to_open("09:31:00", "13:00:00")
    maxpT1330 = oneday_price.maxprice_to_open("09:31:00", "13:30:00")
    maxpT1400 = oneday_price.maxprice_to_open("09:31:00", "14:00:00")
    maxpT1430 = oneday_price.maxprice_to_open("09:31:00", "14:30:00")

    minpT1000 = oneday_price.minprice_to_open("09:31:00", "10:00:00")
    minpT1030 = oneday_price.minprice_to_open("09:31:00", "10:30:00")
    minpT1100 = oneday_price.minprice_to_open("09:31:00", "11:00:00")
    minpT1300 = oneday_price.minprice_to_open("09:31:00", "13:00:00")
    minpT1330 = oneday_price.minprice_to_open("09:31:00", "13:30:00")
    minpT1400 = oneday_price.minprice_to_open("09:31:00", "14:00:00")
    minpT1430 = oneday_price.minprice_to_open("09:31:00", "14:30:00")

    prevClose_prevSettle = oneday_price.close_to_settle()
    OpenPrice = oneday_price.get_close_price("09:31:00", "15:00:00")[0]
    ClosePrice = oneday_price.get_close_price("09:31:00", "15:00:00")[-1]

    train = pd.DataFrame([Date, OpenPrice, ClosePrice, prevClose_prevSettle,
                          start1000, start1030, start1100, start1300, start1330,
                          start1400, start1430, start1500,
                          R1000, R1030, R1100, R1300, R1330, R1400, R1430,
                          maxpT1000, maxpT1030, maxpT1100, maxpT1300, maxpT1330,
                          maxpT1400, maxpT1430,
                          minpT1000, minpT1030, minpT1100, minpT1300, minpT1330,
                          minpT1400, minpT1430]).transpose()
    return train

def get_traindata(TimestampPrice, csv_flag=True):

    if  csv_flag==True:
        ATST_train = pd.read_csv(r'data\ATST_train.csv')
        ATST_train.Date = pd.to_datetime(ATST_train.Date)
    else:
        everyday = split_by_day(TimestampPrice)
        ATST_train = []
        for x in everyday:
            try:
                ATST_train.append(maketrain(x))
            except:
                print(str(x) + "error occured \n")

        ATST_train = pd.concat(ATST_train)
        ATST_train.columns = ['Date', 'OpenPrice', 'ClosePrice',
                              'prevClose_prevSettle',
                              'start1000', 'start1030', 'start1100', 'start1300',
                              'start1330', 'start1400', 'start1430', 'start1500',
                              'R1000', 'R1030', 'R1100', 'R1300', 'R1330', 'R1400',
                              'R1430',
                              'maxpT1000', 'maxpT1030', 'maxpT1100', 'maxpT1300',
                              'maxpT1330', 'maxpT1400', 'maxpT1430',
                              'minpT1000', 'minpT1030', 'minpT1100', 'minpT1300',
                              'minpT1330', 'minpT1400', 'minpT1430']

        # change all the data type to float
        ATST_train.iloc[:, range(1, np.shape(ATST_train)[1])] = ATST_train.iloc[:, range(1, np.shape(ATST_train)[1])].astype(float)
        ATST_train.iloc[:, range(1, np.shape(ATST_train)[1])] = ATST_train.iloc[:, range(1, np.shape(ATST_train)[1])].apply(
            lambda x: pd.to_numeric(x, errors='ignore'))

        ATST_train['prevClose'] = ATST_train['ClosePrice'].shift(1)
        ATST_train['prevClose_prevSettle'] = ATST_train['prevClose_prevSettle'].shift(1)
        ATST_train['prevClose_prevOpen'] = ATST_train['ClosePrice'] / ATST_train['OpenPrice'] - 1
        ATST_train['prevClose_prevOpen'] = ATST_train['prevClose_prevOpen'].shift(1)
        ATST_train['todayOpen_prevClose'] = ATST_train['OpenPrice'] / ATST_train['prevClose'] - 1

        ATST_train = ATST_train.dropna(axis=0)
        print(ATST_train.head())

    # 特征工程
    traindata = []
    traindata.append(ATST_train[['R1000', 'Date', 'start1000',
                                 'maxpT1000', 'minpT1000',
                                 'prevClose_prevSettle', 'prevClose_prevOpen',
                                 'todayOpen_prevClose']])

    traindata.append(ATST_train[['R1030', 'Date', 'start1000', 'start1030',
                                 'maxpT1030', 'minpT1030',
                                 'prevClose_prevSettle', 'prevClose_prevOpen',
                                 'todayOpen_prevClose']])

    traindata.append(
        ATST_train[['R1100', 'Date', 'start1000', 'start1030', 'start1100',
                    'maxpT1100', 'minpT1100',
                    'prevClose_prevSettle', 'prevClose_prevOpen',
                    'todayOpen_prevClose']])

    traindata.append(ATST_train[
                         ['R1300', 'Date', 'start1000', 'start1030',
                          'start1100',
                          'start1300',
                          'maxpT1300', 'minpT1300',
                          'prevClose_prevSettle', 'prevClose_prevOpen',
                          'todayOpen_prevClose']])

    traindata.append(ATST_train[
                         ['R1330', 'Date', 'start1000', 'start1030',
                          'start1100',
                          'start1300', 'start1330',
                          'maxpT1330', 'minpT1330',
                          'prevClose_prevSettle', 'prevClose_prevOpen',
                          'todayOpen_prevClose']])

    traindata.append(ATST_train[
                         ['R1400', 'Date', 'start1000', 'start1030',
                          'start1100',
                          'start1300', 'start1330', 'start1400',
                          'maxpT1400', 'minpT1400',
                          'prevClose_prevSettle', 'prevClose_prevOpen',
                          'todayOpen_prevClose']])

    traindata.append(ATST_train[
                         ['R1430', 'Date', 'start1000', 'start1030',
                          'start1100',
                          'start1300', 'start1330', 'start1400', 'start1430',
                          'maxpT1430', 'minpT1430',
                          'prevClose_prevSettle', 'prevClose_prevOpen',
                          'todayOpen_prevClose']])

    return traindata


def combine_datetime(date, time):
    Datetime = []
    for i in range(len(date)):
        Datetime.append(datetime.datetime.combine(date.iloc[i], time.iloc[i]))
    return Datetime

def add_time(prediction, time):
    prediction['Time'] = [time] * prediction.shape[0]
    return prediction

def get_TimestampSignal(traindata, start_date = '2018-10-08'):
    # 模型训练
    h2o.init()
    TimestampSignal = []
    for ATST_traindata in traindata:
        train = H2Odeeplearning(ATST_traindata)
        HiddenLayer = list(range(int(np.floor((ATST_traindata.shape[1] - 2) / 2).item()) * 2, 0, -2))
        Hiddendropout = [0.2] * len(HiddenLayer)
        prediction = train.online_dl_with_autoencoder_once_a_day(start_date, HiddenLayer, Hiddendropout)
        TimestampSignal.append(prediction)

    TimestampSignal[0] = add_time(TimestampSignal[0], datetime.time(10, 0))
    TimestampSignal[1] = add_time(TimestampSignal[1], datetime.time(10, 30))
    TimestampSignal[2] = add_time(TimestampSignal[2], datetime.time(11, 0))
    TimestampSignal[3] = add_time(TimestampSignal[3], datetime.time(11, 30))
    TimestampSignal[4] = add_time(TimestampSignal[4], datetime.time(13, 30))
    TimestampSignal[5] = add_time(TimestampSignal[5], datetime.time(14, 0))
    TimestampSignal[6] = add_time(TimestampSignal[6], datetime.time(14, 30))
    TimestampSignal = pd.concat(TimestampSignal)
    TimestampSignal = TimestampSignal.sort_values(by=['Date', 'Time'])
    TimestampSignal['DATETIME'] = combine_datetime(TimestampSignal['Date'],
                                                   TimestampSignal['Time'])
    print(TimestampSignal.head())
    TimestampSignal.to_csv('data/TimestampSignal.csv', index=None)

    return TimestampSignal

def get_backtest_result(TimestampPrice,  TimestampSignal=pd.DataFrame()):

    if TimestampSignal.empty:
        TimestampSignal = pd.read_csv('data/TimestampSignal.csv')
        TimestampSignal.Date = pd.to_datetime(TimestampSignal.Date, format='%Y-%m-%d')
        TimestampSignal.Date = TimestampSignal.Date.apply(lambda x: x.date())
        TimestampSignal.DATETIME = pd.to_datetime(TimestampSignal.DATETIME)
        #the time period for TimestampPrice must be bigger than TimestampSignal
        backtestMins = backtest_minutes(TimestampSignal, TimestampPrice,
                                        opencost=2.5 / 10000, closecost=2.5 / 10000, closecost2=23 / 10000)
        datevalues = backtestMins.crossday_fixedstoploss_mins_value_everyday(r=0.04)
        datevalues.to_csv('data/datevalues.csv', index=None)


    ReturnsManipulation = Returns_calculation(datevalues)
    # to merge same day return
    ReturnsManipulation.merge_by_date()
    # to impute zeros for days without trading
    tradedates = TimestampSignal['Date'].unique()
    ReturnsManipulation.impute_zero_returns(tradedates)
    # calculate the simple returns
    ReturnsManipulation.returns_to_Simplevalues()

    # combine backtest result with dates
    date_val = combine_date_value(pd.Series(tradedates),
                                  ReturnsManipulation.SimpleValue)
    date_val.date_and_value(include_first=True)

    datevalues = date_val.datevalue
    datevalues.columns = ['Date', 'Value']
    datevalues.set_index(['Date'], inplace=True)

    ### plot feature importance
    fig, ax = plt.subplots(figsize=(15, 15))
    datevalues['Value'].plot()
    plt.show()

if __name__ == '__main__':
    TimestampPrice = pd.read_pickle('./data/TimestampPriceMin.pickle')
    # 获得训练数据
    traindata = get_traindata(TimestampPrice)
    # # 获取开仓信号
    TimestampSignal = get_TimestampSignal(traindata, start_date = '2016-01-01')
    # 获得回测结果
    # import tushare as ts
    # s1 = ts.get_k_data('000001', index=True)
    # test = s1.columns.difference(['code','close'])
    # print(test)
    get_backtest_result(TimestampPrice)

# #    df_train = pd.read_csv(r'data\ATST_train.csv')
#      #
#      #
#     everyday = split_by_day(TimestampPrice)
#
#     #每个element是每日行情数据，通过maketrain计算每天的训练数据
#     #得到所有的训练数据
#     ATST_train = []
#     for x in everyday:
#         try:
#             ATST_train.append(maketrain(x))
#         except:
#             print(str(x) + "error occured \n")
#
#     ATST_train = pd.concat(ATST_train)
#     ATST_train.columns = ['Date', 'OpenPrice', 'ClosePrice', 'prevClose_prevSettle',
#                           'start1000', 'start1030', 'start1100', 'start1300', 'start1330', 'start1400', 'start1430',
#                           'start1500',
#                           'R1000', 'R1030', 'R1100', 'R1300', 'R1330', 'R1400', 'R1430',
#                           'maxpT1000', 'maxpT1030', 'maxpT1100', 'maxpT1300', 'maxpT1330', 'maxpT1400', 'maxpT1430',
#                           'minpT1000', 'minpT1030', 'minpT1100', 'minpT1300', 'minpT1330', 'minpT1400', 'minpT1430']
#
#     # change all the data type to float
#     ATST_train.dtypes
#     ATST_train.iloc[:, range(1, np.shape(ATST_train)[1])] = ATST_train.iloc[:, range(1, np.shape(ATST_train)[1])].astype(
#         float)
#     ATST_train.iloc[:, range(1, np.shape(ATST_train)[1])] = ATST_train.iloc[:, range(1, np.shape(ATST_train)[1])].apply(
#         lambda x: pd.to_numeric(x, errors='ignore'))
#
#     ATST_train['prevClose'] = ATST_train['ClosePrice'].shift(1)
#     ATST_train['prevClose_prevSettle'] = ATST_train['prevClose_prevSettle'].shift(1)
#     ATST_train['prevClose_prevOpen'] = ATST_train['ClosePrice'] / ATST_train['OpenPrice'] - 1
#     ATST_train['prevClose_prevOpen'] = ATST_train['prevClose_prevOpen'].shift(1)
#     ATST_train['todayOpen_prevClose'] = ATST_train['OpenPrice'] / ATST_train['prevClose'] - 1
#
#     ATST_train = ATST_train.dropna(axis=0)
#
#     t1 = time.time()
#      ATST_train = pd.read_csv(r'data\ATST_train.csv')
#      ATST_train.Date = pd.to_datetime(ATST_train.Date)
#
#     traindata=[]
#     traindata.append(ATST_train[['R1000','Date','start1000',
#     'maxpT1000','minpT1000',
#     'prevClose_prevSettle','prevClose_prevOpen','todayOpen_prevClose']])
#
#     traindata.append(ATST_train[['R1030','Date','start1000','start1030',
#     'maxpT1030','minpT1030',
#     'prevClose_prevSettle','prevClose_prevOpen','todayOpen_prevClose']])
#
#     traindata.append(ATST_train[['R1100','Date','start1000','start1030','start1100',
#     'maxpT1100','minpT1100',
#     'prevClose_prevSettle','prevClose_prevOpen','todayOpen_prevClose']])
#
#     traindata.append(ATST_train[['R1300','Date','start1000','start1030','start1100','start1300',
#     'maxpT1300','minpT1300',
#     'prevClose_prevSettle','prevClose_prevOpen','todayOpen_prevClose']])
#
#     traindata.append(ATST_train[['R1330','Date','start1000','start1030','start1100','start1300','start1330',
#     'maxpT1330','minpT1330',
#     'prevClose_prevSettle','prevClose_prevOpen','todayOpen_prevClose']])
#
#     traindata.append(ATST_train[['R1400','Date','start1000','start1030','start1100','start1300','start1330','start1400',
#     'maxpT1400','minpT1400',
#     'prevClose_prevSettle','prevClose_prevOpen','todayOpen_prevClose']])
#
#     traindata.append(ATST_train[['R1430','Date','start1000','start1030','start1100','start1300','start1330','start1400','start1430',
#     'maxpT1430','minpT1430',
#     'prevClose_prevSettle','prevClose_prevOpen','todayOpen_prevClose']])
#
#     h2o.init()
#     TimestampSignal = []
#     for ATST_traindata in traindata:
#         train = H2Odeeplearning(ATST_traindata)
#         HiddenLayer = list(range(int(np.floor((ATST_traindata.shape[1] - 2) / 2).item()) * 2, 0, -2))
#         Hiddendropout = [0.2] * len(HiddenLayer)
#         prediction = train.online_dl_with_autoencoder_once_a_day('2018-11-01', HiddenLayer, Hiddendropout)
#         TimestampSignal.append(prediction)
#
#     print(time.time() - t1)
#
#     TimestampSignal[0] = add_time(TimestampSignal[0], datetime.time(10, 0))
#     TimestampSignal[1] = add_time(TimestampSignal[1], datetime.time(10, 30))
#     TimestampSignal[2] = add_time(TimestampSignal[2], datetime.time(11, 0))
#     TimestampSignal[3] = add_time(TimestampSignal[3], datetime.time(11, 30))
#     TimestampSignal[4] = add_time(TimestampSignal[4], datetime.time(13, 30))
#     TimestampSignal[5] = add_time(TimestampSignal[5], datetime.time(14, 0))
#     TimestampSignal[6] = add_time(TimestampSignal[6], datetime.time(14, 30))
#     TimestampSignal = pd.concat(TimestampSignal)
#     TimestampSignal = TimestampSignal.sort_values(by=['Date', 'Time'])
#     TimestampSignal['DATETIME'] = combine_datetime(TimestampSignal['Date'], TimestampSignal['Time'])
#
#     # the time period for TimestampPrice must be bigger than TimestampSignal
#     backtestMins=backtest_minutes(TimestampSignal,TimestampPrice,opencost=2.5/10000,closecost=2.5/10000,closecost2=23/10000)
#     datevalues=backtestMins.crossday_fixedstoploss_mins_value_everyday(r=0.04)
#
#     datevalues.to_csv('data/datevalues.csv', index=None)
#     TimestampSignal.to_csv('data/TimestampSignal.csv', index=None)
#
#     '''
#     直接读取datevalues, TimestampSignal
# #    '''
#     datevalues = pd.read_csv('data/datevalues.csv')
#     TimestampSignal = pd.read_csv('data/TimestampSignal.csv')
#     TimestampSignal.Date = pd.to_datetime(TimestampSignal.Date, format='%Y-%m-%d')
#     TimestampSignal.Date = TimestampSignal.Date.apply(lambda x:x.date())
#
#
#    ReturnsManipulation=Returns_calculation(datevalues)
#    # to merge same day return
#    ReturnsManipulation.merge_by_date()
#    # to impute zeros for days without trading
#    tradedates=TimestampSignal['Date'].unique()
#    ReturnsManipulation.impute_zero_returns(tradedates)
#    # calculate the simple returns
#    ReturnsManipulation.returns_to_Simplevalues()
#    
#    # combine backtest result with dates
#    date_val=combine_date_value(pd.Series(tradedates),ReturnsManipulation.SimpleValue)
#    date_val.date_and_value(include_first=True)
#    
#    datevalues=date_val.datevalue
#    datevalues.columns=['Date','Value']
#    datevalues.set_index(['Date'],inplace=True)
#    datevalues['Value'].plot()
#    
#    import matplotlib.pyplot as plt
#    plt.show()    